# -*- coding: utf-8 -*-
import random
import re

import time
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
from scrapy.http import HtmlResponse
from selenium import webdriver
from selenium.webdriver import Proxy, DesiredCapabilities
from selenium.webdriver.common.proxy import ProxyType
from taobao_category import settings
from taobao_category.config.ip_pool import ips_pool
from taobao_category.config.user_agents import user_agents_pool
from taobao_category.models import BShop
from taobao_category.settings_local import taobao_info
from taobao_category.sqlalchemy_engine import DBSession
from taobao_category.utils.ips_proxy_utils import get_ips_pool
from taobao_category.utils.utils import parse_url


class CustIpProxySMiddleware(HttpProxyMiddleware):
    def __init__(self,ip=''):
        '''初始化'''
        self.ip=ip
        self.ip_pools = ips_pool
    def process_request(self, request, spider):
        '''使用代理ip，随机选用'''
        host=random.choice(self.ip_pools) #随机选择一个ip
        # print '当前使用的IP是'+host['ip']
        try:
            request.meta["proxy"] = "http://"+host['ip']+":"+host['port']
        except Exception,e:
            print e
            pass

class IpProxysMiddleware(object):
    def __init__(self,ip=''):
        '''http或https代理'''
        self.ip=ip
        self.ip_proxy_pools = get_ips_pool()
    def process_request(self, request, spider):
        '''使用代理ip，随机选用'''
        # print '当前使用的IP是'+host['ip']
        try:
            if request.meta.get('is_proxy', 0):
                print '使用代理中间件'
                protocol, host, _ = parse_url(request.url)
                protocol_proxy = self.ip_proxy_pools.get(protocol, [])
                proxy = random.choice(protocol_proxy)
                request.meta["proxy"] = proxy
        except Exception,e:
            print e
            pass

    def process_response(self, request, response, spider):
        '''
        如果代理服务器不响应
        :param request: 
        :param response: 
        :param spider: 
        :return: 
        '''
        # print 1
        return response

class CustUserAgentMiddleware(UserAgentMiddleware):
    def __init__(self,user_agent=''):
        self.user_agent=user_agent
        self.user_agent_pools = user_agents_pool
    def process_request(self, request, spider):
        '''使用代理UA，随机选用'''
        ua=random.choice(self.user_agent_pools)
        # print '当前使用的user-agent是'+ua
        try:
            request.headers.setdefault('User-Agent',ua)
        except Exception,e:
            print e
            pass

class CustHeadersMiddleware(object):

    def __init__(self):
        self.user_agent_pools = user_agents_pool
        self.headers = settings.DEFAULT_REQUEST_HEADERS

    def process_request(self, request, spider):

        try:
            ua = random.choice(self.user_agent_pools)
            # print '当前使用的user-agent是'+ua
            request.headers.setdefault('User-Agent', ua)
            request.headers.update(self.headers)
        except Exception, e:
            print e
            pass

class SeleniumMiddleware(object):

    def __init__(self):
        self.driver = webdriver.PhantomJS()  # 指定使用的浏览器
        # driver = webdriver.Firefox()
        # driver = webdriver.Chrome()
        self.allowed_status = (301, 302, 303, 307, 308)
        self.session = DBSession()
        self.ip_proxy_pools = get_ips_pool()
        self.pattern = re.compile('shop(.*?).taobao.com', re.S | re.I)

    def process_request(self, request, spider):
        is_dynamic_page = request.meta.get('is_dynamic_page', False)
        if is_dynamic_page:
            # print "PhantomJS is starting..."
            # protocol, host, _ = parse_url(request.url)
            # protocol_proxy = self.ip_proxy_pools.get(protocol, [])
            # http_proxy = random.choice(protocol_proxy)
            # http_protocol, http_host, http_path = parse_url(http_proxy)
            # PhantomJSproxy = Proxy()
            # PhantomJSproxy.proxy_type = ProxyType.MANUAL
            # PhantomJSproxy.http_proxy = http_host
            # # 再新建一个“期望技能”，（）
            # desired_capabilities = DesiredCapabilities.PHANTOMJS.copy()
            # # 把代理ip加入到技能中
            # PhantomJSproxy.add_to_capabilities(desired_capabilities)
            # # 新建一个会话，并把技能传入
            # self.driver.start_session(desired_capabilities)

            self.driver.get(request.url)
            time.sleep(1)
            js = "var q=document.documentElement.scrollTop=10000"
            self.driver.execute_script(js) #可执行js，模仿用户操作。此处为将页面拉至最底端。
            time.sleep(5)
            body = self.driver.page_source
            # print ("访问"+request.url)
            return HtmlResponse(self.driver.current_url, body=body, encoding='utf-8', request=request)
        else:
            return

    def process_response(self, request, response, spider):
        # 针对b店，若直接爬取不了就模拟浏览器的行为爬取页面
        result = re.search(self.pattern, response.url)
        if response.status in self.allowed_status and result:
            bshop = BShop(shop_id='%s' % request.meta['shop_id'])
            self.session.add(bshop)
            self.session.commit()

            self.driver.get(request.url)
            time.sleep(4)
            js = "var q=document.documentElement.scrollTop=10000"
            self.driver.execute_script(js)  # 可执行js，模仿用户操作。此处为将页面拉至最底端。
            time.sleep(1)
            body = self.driver.page_source
            # print ("访问"+request.url)
            return HtmlResponse(self.driver.current_url, body=body, encoding='utf-8', request=request)
        return response

class AutologonMiddleware(object):

    def __init__(self):
        self.protocal, self.host, self.path = parse_url(taobao_info['login_url'])

    def process_response(self, request, response, spider):
        pass

class TmallShopInterceptMiddleware(object):

    def __init__(self):
        self.allowed_status = (301, 302, 303, 307, 308)
        self.session = DBSession()
        self.pattern = re.compile('shop(.*?).taobao.com', re.S|re.I)


    def process_response(self, request, response, spider):
        result = re.search(self.pattern, response.url)
        if response.status in self.allowed_status and result:
            response.status = 999
            bshop = BShop(shop_id='%s' % request.meta['shop_id'])
            self.session.add(bshop)
            self.session.commit()
            print 1

        return response


# class TaobaoLoginSimulate(object):
#     '''
#     淘宝网登陆模拟
#     '''
#
#     def process_response(request, response, spider):
#         '''
#         :param response:
#         :param spider:
#         :return:
#         '''
#
#     def process_request(self, request, spider):
#         protocol, host, _ = parse_url(request.url)
#         if host == 'login.taobao.com'
#         is_dynamic_page = request.meta.get('is_dynamic_page', False)
#         if is_dynamic_page:
#             # print "PhantomJS is starting..."
#             driver = webdriver.PhantomJS() #指定使用的浏览器
#             # driver = webdriver.Firefox()
#             # driver = webdriver.Chrome()
#             driver.get(request.url)
#             time.sleep(1)
#             js = "var q=document.documentElement.scrollTop=10000"
#             driver.execute_script(js) #可执行js，模仿用户操作。此处为将页面拉至最底端。
#             time.sleep(10)
#             body = driver.page_source
#             # print ("访问"+request.url)
#             return HtmlResponse(driver.current_url, body=body, encoding='utf-8', request=request)
#         else:
#             return


